import numpy as np
import pandas as pd
from num2words import num2words

# sklearn
from sklearn import metrics
from sklearn.model_selection import cross_val_score, KFold, StratifiedShuffleSplit, ShuffleSplit
from sklearn.datasets import make_moons, make_circles, make_classification, load_digits
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.preprocessing import scale

# Tensorflow
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

# Visualisation libraries

## progressbar
import progressbar

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
from matplotlib.font_manager import FontProperties
import matplotlib.colors as mcolors
from matplotlib.colors import LinearSegmentedColormap
from matplotlib import cm
plt.style.use('seaborn-whitegrid')
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")

Home / Deep Learning

TensorFlow - Keras

• Keras Multi-layer Perceptron (MLP) for Binary Classifications
• Keras Multi-layer Perceptron (MLP) for Multi-Class Classifications
• Keras Multi-layer Perceptron (MLP) for Image Classifications
• Keras Multi-layer Perceptron (MLP) for Regression

In this article, we go through basic image classification using Multi-layer Perceptron (MLP). For testing the algorithm, we use sklearn digit dataset.

Dataset

fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(17, 4), subplot_kw={'xticks': [], 'yticks': []})
ax = ax.ravel()
font = FontProperties()
font.set_family('fantasy')

digits = load_digits()
for ax, (image, label) in zip(ax, list(zip(digits.images, digits.target))):
    _ = ax.imshow(image, cmap= 'Greys', interpolation='nearest')
    _ = ax.set_title('%s' % num2words(label), fontproperties=font, fontsize = 16)
    _ = ax.set_aspect(1)
X = digits.images/ 256.0
y = digits.target 
Labels_dict = dict(zip(list(np.unique(y)), [num2words(x).title() for x in np.unique(y)]))

Train and Test Sets

def Dist_Plot(y, PD, Labels_dict = Labels_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths= PD['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])
    
    # Right
    y = ToSeries(y)
    fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                         values= y.value_counts().values, pull=[0.1],
                         textfont=dict(size=PD['textfont']),
                         marker=dict(colors = PD['PieColors'],
                         line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=.5)
    fig.update_layout(legend=dict(orientation="v"), legend_title_text= PD['legend_title'])
    
    # Left
    
    ## Table
    Table = y.value_counts().to_frame('Count').reset_index(drop = False)
    Table = Table.rename(columns = {'index':PD['legend_title']}).sort_values(by = [PD['legend_title']])
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)

    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%% %.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    TableColors = PD['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = PD['tablecolumnwidth'],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not PD['height'] == None:
        fig.update_layout(height = PD['height'])
    fig.show()
    
PD = dict(PieColors = px.colors.sequential.deep, column_widths = [0.3, 0.7], tablecolumnwidth = [0.25, 0.15, 0.2],
          textfont = 12, TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Dist_Plot(y, PD)

Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
    # X
    if isinstance(X, pd.DataFrame):
        X_train, X_test = X.loc[train_index], X.loc[test_index]
    else:
        X_train, X_test = X[train_index], X[test_index]
    # y    
    if isinstance(y, pd.Series):
        y_train, y_test = y[train_index], y[test_index]
    else:
        y_train, y_test = y[train_index], y[test_index]
del sss

def Train_Test_Dist(X_train, y_train, X_test, y_test, PD):
    def ToSeries(x):
        if not isinstance(x, pd.DataFrame):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= PD['column_widths'],
                        specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
    
    y_train = ToSeries(y_train)
    y_test = ToSeries(y_test)
    # Right
    C = 2
    for y in [y_train, y_test]:
        fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                             values= y.value_counts().values, pull=[0.1],
                             textfont=dict(size=PD['textfont']),
                             marker=dict(colors = PD['PieColors'],
                             line=dict(color='black', width=1))), row=1, col=C)
        fig.update_traces(hole=.5)
        fig.update_layout(legend=dict(orientation="v"), legend_title_text= PD['legend_title'])
        C+=1
    
    # Left
    
    # Table
    Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
                       'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
    T = Table.copy()
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
        
    TableColors = PD['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not PD['height'] == None:
        fig.update_layout(height = PD['height'])
    fig.show()
    
PD.update(dict(column_widths = [0.3, 0.3, 0.3]))
Train_Test_Dist(X_train, y_train, X_test, y_test, PD)

#
y_train = keras.utils.to_categorical(y_train, num_classes=len(Labels_dict))
y_test = keras.utils.to_categorical(y_test, num_classes=len(Labels_dict))

Image Classification

The goal of this approach is to classify the images by focusing on the relationship between nearby pixels. A simple implementation of an image classifier can be performed in Keras using Multi-layer Perceptron (MLP) Image Classification as follows.

model = keras.Sequential(name = 'MLP_Image_Classification')
model.add(layers.Flatten(input_shape=X[0].shape, name='Layer1'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(32, activation='relu', name='Layer2'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(len(Labels_dict), activation='softmax', name='Layer3'))
#
model.summary()
tf.keras.utils.plot_model(model, show_shapes=True, show_layer_names=True, expand_nested = True, rankdir = 'LR')

Model: "MLP_Image_Classification"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Layer1 (Flatten)             (None, 64)                0         
_________________________________________________________________
dropout (Dropout)            (None, 64)                0         
_________________________________________________________________
Layer2 (Dense)               (None, 32)                2080      
_________________________________________________________________
dropout_1 (Dropout)          (None, 32)                0         
_________________________________________________________________
Layer3 (Dense)               (None, 10)                330       
=================================================================
Total params: 2,410
Trainable params: 2,410
Non-trainable params: 0
_________________________________________________________________

# Number of iterations
IT = int(1e2)+1

model.compile(optimizer= 'rmsprop', loss='binary_crossentropy', metrics=['accuracy', tf.keras.metrics.Recall()])

# Training model
history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs= IT, verbose = 0)

def Search_List(Key, List): return [s for s in List if Key in s]

Metrics_Names = {'loss':'Loss', 'accuracy':'Accuracy', 'mae':'MAE', 'mse':'MSE', 'recall': 'Recall'}

def Table_modify(df, Metrics_Names = Metrics_Names):
    df = df.rename(columns = Metrics_Names)
    df = df.reindex(sorted(df.columns), axis=1)
    df.insert(loc = 0, column = 'Iteration', value = np.arange(0, df.shape[0]), allow_duplicates=False)
    return df

Validation_Table = Search_List('val_',history.history.keys()) 
Train_Table = list(set( history.history.keys()) - set(Validation_Table))
Validation_Table = pd.DataFrame(np.array([history.history[x] for x in Validation_Table]).T, columns = Validation_Table)
Train_Table = pd.DataFrame(np.array([history.history[x] for x in Train_Table]).T, columns = Train_Table)
Validation_Table.columns = [x.replace('val_','') for x in Validation_Table.columns]

Train_Table = Table_modify(Train_Table)
Validation_Table = Table_modify(Validation_Table)

# Train Set Score
score = model.evaluate(X_train, y_train, batch_size=128, verbose = 0)
score = pd.DataFrame(score, index = model.metrics_names).T
score.index = ['Train Set Score']
# Validation Set Score
Temp = model.evaluate(X_test, y_test, batch_size=128, verbose = 0) 
Temp = pd.DataFrame(Temp, index = model.metrics_names).T
Temp.index = ['Validation Set Score']
score = score.append(Temp)
score.rename(columns= Metrics_Names, inplace = True)
score = score.reindex(sorted(score.columns), axis=1)
display(score.style.set_precision(4))

	Accuracy	Loss	Recall
Train Set Score	0.8807	0.1135	0.6189
Validation Set Score	0.8593	0.1178	0.6019

Let's define some function by which we can analyze the performance of the modeling.

def Plot_history(history, Title = False, Table_Rows = 25):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].values,
                             line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].values,
                             line=dict(color='MidnightBlue', width= 1.5),  name = 'Accuracy'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Recall'].values,
                             line=dict(color='SeaGreen', width= 1.5), name = 'Recall'), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, 1], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    ind = np.linspace(0, history.shape[0], Table_Rows, endpoint = False).round(0).astype(int)
    ind = np.append(ind, history.index[-1])
    history = history[history.index.isin(ind)]
    T = history.copy()
    T[['Accuracy','Loss','Recall']] = T[['Accuracy','Loss','Recall']].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
                                       fill_color='DimGray', align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray', fill=dict(color=['WhiteSmoke', 'white']),
                                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    if Title != False:
        fig.update_layout(plot_bgcolor= 'white',
                      title={'text': Title, 'x':0.46, 'y':0.94, 'xanchor': 'center', 'yanchor': 'top'},
                          yaxis_title='Frequency')
    fig.show()

Plot_history(Train_Table, Title = 'Train Set')
Plot_history(Validation_Table, Title = 'Validation Set')

Confusion Matrix

The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.

def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
    if n_splits == None:
        Titles = ['Train Set', 'Test Set']
    else:
        Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
    CM = [CM_Train, CM_Test]
    Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
    for i in range(2):
        fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
        fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
        _ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
                        linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
        _ = ax[0].set_title('Confusion Matrix');
        Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
        _ = sns.heatmap(Temp,
                        annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
                       linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
        _ = ax[1].set_title('Normalized Confusion Matrix');

        for a in ax:
            _ = a.set_xlabel('Predicted labels')
            _ = a.set_ylabel('True labels'); 
            _ = a.xaxis.set_ticklabels(PD['Labels'])
            _ = a.yaxis.set_ticklabels(PD['Labels'])
            _ = a.set_aspect(1)

# Train
y_pred = model.predict(X_train).argmax(axis = 1)
Reports_Train = pd.DataFrame(metrics.classification_report(y_train.argmax(axis = 1),
                                                           y_pred, target_names=list(Labels_dict.values()),
                                                           output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train.argmax(axis = 1), y_pred)
# Test
y_pred = model.predict(X_test).argmax(axis = 1)
Reports_Test = pd.DataFrame(metrics.classification_report(y_test.argmax(axis = 1),
                                                          y_pred, target_names=list(Labels_dict.values()),
                                                          output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test.argmax(axis = 1), y_pred)

Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
                                                 
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
        set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
        set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))

PD = dict(FS = (16, 6), annot_kws = 10, shrink = .6, Labels = list(Labels_dict.values()))
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)

Train Set	precision	recall	f1-score	support
Zero	0.991935	0.991935	0.991935	124.000000
One	0.795455	0.826772	0.810811	127.000000
Two	0.885496	0.935484	0.909804	124.000000
Three	0.800000	0.875000	0.835821	128.000000
Four	0.968254	0.960630	0.964427	127.000000
Five	0.950000	0.897638	0.923077	127.000000
Six	0.968992	0.984252	0.976562	127.000000
Seven	0.820000	0.984000	0.894545	125.000000
Eight	0.934211	0.581967	0.717172	122.000000
Nine	0.744186	0.761905	0.752941	126.000000
accuracy	0.880668	0.880668	0.880668	0.880668
macro avg	0.885853	0.879958	0.877710	1257.000000
weighted avg	0.885557	0.880668	0.878038	1257.000000

Test Set	precision	recall	f1-score	support
Zero	1.000000	0.962963	0.981132	54.000000
One	0.672414	0.709091	0.690265	55.000000
Two	0.803279	0.924528	0.859649	53.000000
Three	0.892857	0.909091	0.900901	55.000000
Four	0.896552	0.962963	0.928571	54.000000
Five	0.938776	0.836364	0.884615	55.000000
Six	0.962963	0.962963	0.962963	54.000000
Seven	0.803030	0.981481	0.883333	54.000000
Eight	1.000000	0.538462	0.700000	52.000000
Nine	0.741379	0.796296	0.767857	54.000000
accuracy	0.859259	0.859259	0.859259	0.859259
macro avg	0.871125	0.858420	0.855929	540.000000
weighted avg	0.870571	0.859259	0.856329	540.000000

Predictions

Pred = model.predict(X_test)
def Pred_Plot(i, Pred = Pred, y_test = y_test, Labels_dict = Labels_dict):
    Pred_Labels = np.argmax(Pred, axis = 1)
    True_Labels = np.argmax(y_test, axis =1)
    Img = X_test[i]*256
    Prop = Pred[i]
    fig, ax = plt.subplots(1, 2, figsize=(16, 5))
    # Left
    _ = ax[0].imshow(Img, cmap=plt.cm.binary)
    _ = ax[0].set_axis_off()
    _ = ax[0].set_aspect(1)

    if Pred_Labels[i] == True_Labels[i]:
        C1 = 'HoneyDew'
        C2 = 'LimeGreen'
        EC = 'DarkGreen'
    else:
        C1 = 'MistyRose'
        C2 = 'OrangeRed'
        EC = 'DarkRed'
    _ = fig.suptitle('Predicted Label = %i, True Label = %i, Accuracy = %% %.2f' %
                     (Pred_Labels[i], True_Labels[i], 100*Pred.max(axis = 1)[i]), color = EC, fontsize = 14)
    # Right
    _ = sns.barplot(ax = ax[1], x= list(Labels_dict.values()), y= Prop,
                    palette=[C1 if (x < max(Prop)) else C2 for x in Prop],
#                     palette='RdBu',
                    edgecolor= EC,  hatch="///")
    _ = ax[1].set_yscale('log')
    _ = ax[1].set_ylim(top = 1)
    
Pred_Labels = np.argmax(Pred, axis = 1)
True_Labels = np.argmax(y_test, axis =1)
# redicted Labels (Correct)
C = np.where(abs(Pred_Labels - True_Labels) == 0)[0]
# Predicted Labels (Wrong)
W = np.nonzero(abs(Pred_Labels - True_Labels))[0]

For example, we can randomly pick an entry from the correctly predicted labels list

Pred_Plot(np.random.choice(C))

Similarly, from the wrongly predicted labels list

Pred_Plot(np.random.choice(W))

---

References

1. scikit-learn digits dataset
2. Keras developer guides
3. Multilayer perceptron wikipedia page
4. Confusion matrix wikipedia page

---